import datetime
from jili.data.db import getdb_client,get_calender_a
from jili.tool.convert import str2datetime
import re
from tqdm import tqdm
from jili.core import save,load
from jili.core.printlog import print
from copy import deepcopy
from research.factor import end_date_calcor,gen_tradedata_a
def top10_holders_floatholders():
    db = getdb_client("stock")
    d1={}
    d2={}
    keys={}
    c1 = end_date_calcor()
    c2 = end_date_calcor()
    for i in tqdm(db["stock_other_top10_floatholders"].find({"end_date":{"$gte":"20100630"}})):
        del i["_id"]
        c1.ondata(i)
        obj=i["obj"]
        ann_date=i["ann_date"]
        end_date=i["end_date"]
        if obj not in d1.keys():
            d1[obj]={}
        if end_date not in d1[obj].keys():
            d1[obj][end_date]=[]
        d1[obj][end_date].append(i)

        if obj not in keys.keys():
            keys[obj]=[ann_date]
        else:
            if ann_date not in keys[obj]:
                keys[obj].append(ann_date)

    for i in tqdm(db["stock_other_top10_holders"].find({"end_date":{"$gte":"20100630"}})):
        del i["_id"]
        c2.ondata(i)
        obj = i["obj"]
        ann_date = i["ann_date"]
        end_date = i["end_date"]
        if obj not in d2.keys():
            d2[obj] = {}
        if end_date not in d2[obj].keys():
            d2[obj][end_date] = []
        d2[obj][end_date].append(i)

        if obj not in keys.keys():
            keys[obj] = [ann_date]
        else:
            if ann_date not in keys[obj]:
                keys[obj].append(ann_date)
    c1.calc_lastend_date()
    c2.calc_lastend_date()
    d3={}
    for obj,v in tqdm(c2.lastend_date.items()):
        d3[obj]={}
        t2=d2[obj]
        for ann_date,end_date in v.items():
            d3[obj][ann_date] = []
            t=t2[end_date]
            if end_date[-4:] not in ["1231","0331","0630","0930"]:
                if len(t)<10:
                    l=list(t2.keys())
                    l.sort()
                    preend_date=None
                    for dd in l:
                        if dd<end_date:
                            preend_date=dd
                        else:
                            break
                    if preend_date:
                        tdict={}
                        for i in t:
                            name=i["holder_name"]
                            tdict[name]=i
                        for i in t2[preend_date]:
                            name=i["holder_name"]
                            if name in tdict.keys():
                                i.update(tdict[name])
                            d3[obj][ann_date].append(i)
                    else:
                        d3[obj][ann_date] = t
                else:
                    d3[obj][ann_date]=t
            else:
                d3[obj][ann_date]=t
    data={}
    for obj,v in tqdm(keys.items()):
        v.sort()
        t1=d1[obj]
        t2=d2[obj]
        data[obj]={}
        for ann_date in v:
            data[obj][ann_date]=[]
            end2=c2.get_lastend_date(obj,ann_date)
            if end2 in t1.keys():
                l=list(t1.keys())
                preend_date=None
                for dd in l:
                    if dd<end2:
                        preend_date=dd
                    else:
                        break
                tdict = {}
                for i in t2[end2]:
                    name = i["holder_name"]
                    tdict[name] = i
                    data[obj][ann_date].append(i)
                if preend_date:
                    for i in t1[preend_date]:
                        name = i["holder_name"]
                        if name not in tdict.keys():
                            data[obj][ann_date].append(i)
            else:
                tdict = {}
                for i in t2[end2]:
                    name = i["holder_name"]
                    tdict[name] = i
                    data[obj][ann_date].append(i)
                for i in t1[end2]:
                    name = i["holder_name"]
                    if name not in tdict.keys():
                        data[obj][ann_date].append(i)
    return data
    db.client.close()
def deal2():
    db = getdb_client("stock", "127.0.0.1")
    fund_hlist={}
    for i in db["fund_report_portfolio"].find({}):
        del i["_id"]
        fund_code=i["ts_code"]
        if i["symbol"][-3:]!=".HK":
            obj=i["symbol"][:6]
            ann_date=i["ann_date"]
            end_date=i["end_date"]
            if fund_code not in fund_hlist.keys():
                fund_hlist[fund_code]={}
            if end_date not in fund_hlist[fund_code].keys():
                fund_hlist[fund_code][end_date]=[]
            fund_hlist[fund_code][end_date].append(i)
    rst=[]
    objs=[]
    for code,v in tqdm(fund_hlist.items()):
        for date,t in v.items():
            keys=[]
            keys1=[]
            for i in t:
                key=i["symbol"]
                key1 = i["symbol"] + i["end_date"] + str(i["amount"])
                #key1 = i["symbol"] + i["ann_date"] + i["end_date"]
                #key1=i["symbol"]+i["ann_date"]+i["end_date"]+str(i["amount"])
                if key1 not in keys1:
                    keys1.append(key1)
                    if key not in keys:
                        keys.append(key)
                    else:
                        rst.append([code,date,key])
                        print(code,date,key)
                        if code not in objs:
                            objs.append(code)

    print(len(rst),len(objs))



    db.client.close()
def deal_fund_hold_save(save_url=r"G:\factor\k1d\jg_holdpos.pkl",ip="127.0.0.1",invest_type_nohave=[],top10_holdname_keys=["证券投资基金","证券基金"]):
    db=getdb_client("stock",ip)
    d1={}
    keys = {}
    fund_info={}
    fund_codes=[]
    for i in db["fund_fullname_info"].find({}):
        del i["_id"]
        fcode=i["ts_code"]
        fund_info[fcode]=i
        if i["FUND_INITIAL"]=="是":
            fund_codes.append(fcode)
    for i in db["fund_info_all"].find({}):
        del i["_id"]
        fcode = i["ts_code"]
        fund_info[fcode].update(i)
    f={}
    print("calc f 去重")
    c=end_date_calcor()
    for i in db["stock_other_top10_floatholders"].find({}):
        obj= i["obj"]
        date=i["ann_date"]
        end_date=i["end_date"]
        c.ondata(i)
        if obj not in f.keys():
            f[obj]={end_date:{i["holder_name"]:i["hold_amount"]}}
        else:
            t=f[obj]
            if end_date not in t.keys():
                t[end_date] = {i["holder_name"]:i["hold_amount"]}
            else:
                if i["holder_name"] not in t[end_date].keys():
                    key=i["holder_name"]
                    t[end_date][key]=i["hold_amount"]
    print("calc d1 dataframe")
    c.calc_lastend_date()
    for obj,v in tqdm(c.ann_end_date.items()):
        for ann_date, v1 in v.items():
            maxend=c.lastend_date[obj][ann_date]
            for i,vvv in f[obj][maxend].items():
                for name in top10_holdname_keys:
                    if name in i:
                        t={"obj":obj,"ann_date":ann_date,"end_date":maxend,"holder_name":i,"hold_amount":vvv}
                        if obj not in d1.keys():
                            d1[obj] = {}
                        if ann_date not in d1[obj].keys():
                            d1[obj][ann_date]=[]
                        d1[obj][ann_date].append(t)

                        if obj not in keys.keys():
                            keys[obj] = []
                        if ann_date not in keys[obj]:
                            keys[obj].append(ann_date)
    print("deal1")
    fund_hlist={}
    obj_ann_date={}
    c0=end_date_calcor(key="ts_code")
    for i in db["fund_report_portfolio"].find({}):
        del i["_id"]
        fund_code=i["ts_code"]
        if i["symbol"][-3:]!=".HK":
            fcode = i["ts_code"]
            if fcode not in fund_codes:
                continue
            obj=i["symbol"][:6]
            ann_date=i["ann_date"]
            end_date=i["end_date"]
            if fund_code not in fund_hlist.keys():
                fund_hlist[fund_code]={}
            if end_date not in fund_hlist[fund_code].keys():
                fund_hlist[fund_code][end_date]=[]
            fund_hlist[fund_code][end_date].append(i)
            c0.ondata(i)
            if obj not in obj_ann_date.keys():
                obj_ann_date[obj] = []
            if ann_date not in obj_ann_date[obj]:
                obj_ann_date[obj].append(ann_date)
            if obj not in keys.keys():
                keys[obj] = []
            if ann_date not in keys[obj]:
                keys[obj].append(ann_date)
    print("计算d2")
    c0.calc_lastend_date()
    fund0={}
    for fund_code, v1 in c0.lastend_date.items():
        ads = list(v1.keys())
        ads.sort()
        fund0[fund_code]=ads
    d2={}
    for obj,v in tqdm(obj_ann_date.items()):
        v.sort()
        d2[obj]={}
        for ann_date in v:
            d2[obj][ann_date]=[]
            for fund_code,ads in fund0.items():
                maxdd=None
                if ads[0]>ann_date:
                    continue
                if ann_date not in ads:
                    for dd in ads:
                        if dd<=ann_date:
                            maxdd=dd
                        else:
                            break
                else:
                    maxdd=ann_date
                if maxdd:
                    delta = str2datetime(ann_date) - str2datetime(maxdd)
                    delta = delta.days
                    if delta <= 180:
                        lastend_date=c0.get_lastend_date(fund_code, maxdd)
                        for i in fund_hlist[fund_code][lastend_date]:
                            if i["symbol"][:6]!=obj:
                                continue
                            d2[obj][ann_date].append(i)

    data={}
    for obj,v in tqdm(keys.items()):
        v.sort()
        if obj in d1.keys():
            t1=d1[obj]
        else:
            t1={}
        if obj in d2.keys():
            t2=d2[obj]
        else:
            t2={}
        data[obj]={}
        for ann_date in v:
            last_date1=None
            data[obj][ann_date]={}
            data[obj][ann_date]["top10_floatholders"]=[]
            tt1=[]
            t1key=list(t1.keys())
            t1key.sort()
            if ann_date not in t1key:
                for i in t1key:
                    if i<=ann_date:
                        last_date1=i
                    else:
                        break
            else:
                last_date1=ann_date
            if last_date1:
                for j in t1[last_date1]:
                    for name in top10_holdname_keys:
                        if name in j["holder_name"]:
                            tt1.append(j)
            last_date2 = None
            data[obj][ann_date]["fund_hold"] = []
            tt2=[]
            t2key = list(t2.keys())
            t2key.sort()
            if ann_date not in t2key:
                for i in t2key:
                    if i <= ann_date:
                        last_date2 = i
                    else:
                        break
            else:
                last_date2=ann_date
            if last_date2:
                for j in t2[last_date2]:
                    fcode=j["ts_code"]
                    if fund_info[fcode]["invest_type"] not in invest_type_nohave:
                        tt2.append(j)
            tt3=[]
            tt4=[]
            for i in tt2:
                fcode = i["ts_code"]
                name=fund_info[fcode]["FUND_FULLNAME"]
                for j in tt1:
                    if name in j["holder_name"]:
                        if i["end_date"]==j["end_date"]:
                            if i["ann_date"]<=j["ann_date"]:
                                if j["holder_name"] not in tt3:
                                    tt3.append(j["holder_name"])
                            else:
                                if fcode not in tt4:
                                    tt4.append(fcode)
                        elif i["end_date"]>j["end_date"]:
                            if j["holder_name"] not in tt3:
                                tt3.append(j["holder_name"])
                        elif i["end_date"]<j["end_date"]:
                            if fcode not in tt4:
                                tt4.append(fcode)
            for i in tt1:
                if i["holder_name"] not in tt3:
                    data[obj][ann_date]["top10_floatholders"].append(i)
            for i in tt2:
                if i["ts_code"] not in tt4:
                    data[obj][ann_date]["fund_hold"].append(i)
    save(data,save_url)
    db.client.close()
def calc_jgfun_holdpos(save_url,mode="0",ip = '192.168.10.44',size_info_url = r"G:\factor\k1d\size_info.pkl"):
    d=load(r"G:\factor\k1d\jg_holdpos.pkl")
    """:cvar
    1.汇总计算机构持股数量：机构为：公募+私募
    2.计算持股占流动市值的比
    3.计算机构个数
    4.计算机构个数扣除被动投资
    5.计算基金经理个数，一个私募产品算一个
    6.计算基金经理个数扣除被动投资，一个私募产品算一个
    "manager_num":0,"manager_num_noindex":0
    """
    db = getdb_client("stock", ip)
    fund_info = {}
    fund_codes = []
    size_info=load(size_info_url)
    for i in db["fund_fullname_info"].find({}):
        del i["_id"]
        fcode = i["ts_code"]
        fund_info[fcode] = i
        if i["FUND_INITIAL"] == "是":
            fund_codes.append(fcode)
    print("fund_codes",len(fund_codes))
    for i in db["fund_info_all"].find({}):
        del i["_id"]
        fcode = i["ts_code"]
        fund_info[fcode].update(i)
    fund_info_manager={}
    for i in db["fund_info_manager"].find({}):
        del i["_id"]
        fcode = i["ts_code"]
        if fcode in fund_info.keys():
            if fcode not in fund_info_manager:
                fund_info_manager[fcode]=[]
            fund_info_manager[fcode].append(i)
    data={}
    print("fund_info_manager")
    for obj,v in tqdm(d.items()):
        data[obj]={}
        for date,v1 in v.items():
            t={"qty":0,"floathold_ratio":0,"freehold_ratio":0,"jg_num":0,"qty_noindex":0,"floathold_ratio_noindex":0,"freehold_ratio_noindex":0,"jg_num_noindex":0}
            tt=[]
            t1=v1["top10_floatholders"]
            t2=v1["fund_hold"]
            for i in t1:
                t["qty"]=t["qty"]+i["hold_amount"]
                t["jg_num"] = t["jg_num"] + 1
                t["jg_num_noindex"] = t["jg_num_noindex"] + 1
            key1s=[]
            key2s=[]
            for i in t2:
                fcode=i["ts_code"]
                if mode=="0":
                    if fcode not in fund_codes:
                        continue
                    key1=i["ts_code"]+i["end_date"]+str(i["amount"])
                    if key1 in key1s:
                        continue
                    else:
                        key1s.append(key1)
                    key2 = i["ts_code"] + i["ann_date"] + i["end_date"]
                    if key2 in key2s:
                        continue
                    else:
                        key2s.append(key2)
                elif mode=="1":
                    key1 = i["ts_code"] + i["end_date"] + str(i["amount"])
                    if key1 in key1s:
                        continue
                    else:
                        key1s.append(key1)
                    key2 = i["ts_code"] + i["ann_date"] + i["end_date"]
                    if key2 in key2s:
                        continue
                    else:
                        key2s.append(key2)
                elif mode=="2":
                    if fcode not in fund_codes:
                        continue
                delta = str2datetime(date) - str2datetime(i["end_date"])
                delta = delta.days
                if delta > 180:
                    continue
                t["qty"] = t["qty"] + i["amount"]
                jg=fund_info[fcode]["management"]
                itype=fund_info[fcode]["invest_type"]
                if jg not in tt:
                    tt.append(jg)
                    t["jg_num"] = t["jg_num"] + 1
                    if itype!="被动指数型":
                        t["jg_num_noindex"] = t["jg_num_noindex"] + 1
                if itype!="被动指数型":
                    t["qty_noindex"] = t["qty_noindex"] + i["amount"]
            data[obj][date]=t
    today = datetime.datetime.today()
    today = str(today)[:10]
    rst=gen_tradedata_a(data,today)
    presize={}
    data={}
    for date,v in tqdm(rst.items()):
        data[date]={}
        if date in size_info.keys():
            for obj,t in v.items():
                if obj in size_info[date].keys():
                    t["floathold_ratio"]=t["qty"]/size_info[date][obj]["float_share"]/10000
                    t["freehold_ratio"] = t["qty"] / size_info[date][obj]["free_share"]/10000
                    t["floathold_ratio_noindex"] = t["qty_noindex"] / size_info[date][obj]["float_share"] / 10000
                    t["freehold_ratio_noindex"] = t["qty_noindex"] / size_info[date][obj]["free_share"] / 10000
                    presize[obj]=size_info[date][obj]
                elif obj in presize.keys():
                    t["floathold_ratio"] = t["qty"] / presize[obj]["float_share"] / 10000
                    t["freehold_ratio"] = t["qty"] / presize[obj]["free_share"] / 10000
                    t["floathold_ratio_noindex"] = t["qty_noindex"] / presize[obj]["float_share"] / 10000
                    t["freehold_ratio_noindex"] = t["qty_noindex"] / presize[obj]["free_share"] / 10000
                data[date][obj]=t
        elif presize:
            for obj,t in v.items():
                if obj in presize.keys():
                    t["floathold_ratio"] = t["qty"] / presize[obj]["float_share"] / 10000
                    t["freehold_ratio"] = t["qty"] / presize[obj]["free_share"] / 10000
                    t["floathold_ratio_noindex"] = t["qty_noindex"] / presize[obj]["float_share"] / 10000
                    t["freehold_ratio_noindex"] = t["qty_noindex"] / presize[obj]["free_share"] / 10000
                data[date][obj] = t
    save(data,save_url)
    db.client.close()
def deal():
    import multiprocessing
    n = multiprocessing.cpu_count() - 2
    if n > 4:
        n = 4
    pool = multiprocessing.Pool(n)
    pool.apply_async(calc_jgfun_holdpos, (r"G:\factor\k1d\jg_hold_info0","0",))
    # pool.apply_async(calc_jgfun_holdpos, (r"G:\factor\k1d\jg_hold_info1", "1",))
    # pool.apply_async(calc_jgfun_holdpos, (r"G:\factor\k1d\jg_hold_info2", "2",))
    # pool.apply_async(calc_jgfun_holdpos, (r"G:\factor\k1d\jg_hold_info3", "3",))
    pool.close()
    pool.join()
if __name__=="__main__":
    #deal()

    calc_jgfun_holdpos(r"G:\factor\k1d\jg_hold_info0","0")
    # calc_jgfun_holdpos(r"G:\factor\k1d\jg_hold_info1", "1")
    # calc_jgfun_holdpos(r"G:\factor\k1d\jg_hold_info2", "2")
    # calc_jgfun_holdpos(r"G:\factor\k1d\jg_hold_info3", "3")












